import pandas as pd


# plotting






import matplotlib.pyplot as plt


from utils import MeanEncoder


# 读入数据
data = pd.read_csv("train_new.csv")
print ("Dataset dimensions:", data.shape)

# 样本数量太多了，运行比较慢，为了作业，先取一部分数据分析，又由于负样本数据实在太多了，正样本太少，所以从负样本中随机采样5000000个

data=data[data['click']==1].append(data[data['click']==0].sample(n=5000000,random_state=10, axis=0))

print ("Dataset dimensions:", data.info())

print ("Dataset dimensions:", data.isnull().any())

# 正负例样本情况
print('正负样本情况',data['click'].value_counts())



Cat_features = data.columns.drop(['id','click','hour'])
for col in Cat_features:
    num_vlaules = len(data[col].unique())
    print ('\n%s属性有%d的不同取值，各取值及其出现的次数\n'% (col,num_vlaules) )
    print (data[col].value_counts())

Cat_features_MEncoder = ['site_id','site_domain','app_id','app_domain','device_id','device_ip','device_model','C14','C17','C20']
Mencoder = MeanEncoder(Cat_features_MEncoder)
data = Mencoder.fit_transform(data, data['click'])
data.drop(columns=['site_id_pred_0','site_domain_pred_0','app_id_pred_0','app_domain_pred_0','device_id_pred_0','device_ip_pred_0','device_model_pred_0','C14_pred_0','C17_pred_0','C20_pred_0'],inplace=True)
data.drop(columns=['id','hour'],inplace=True)
data_MEncoder = data[Cat_features_MEncoder]
data.drop(columns=Cat_features_MEncoder,inplace=True)
data['device_type']=data['device_type'].astype(str)
data['C18'] = data['C18'].astype(str)
data['hour_time'] = data['hour_time'].astype(str)
#print ("after dataencoder:", data.info())

data = pd.get_dummies(data)
data = pd.concat([data,data_MEncoder],axis=1)
data = data.sample(frac=1)

# 样本数量太多了，运行比较慢，为了作业，先取一部分数据分析，又由于负样本数据实在太多了，正样本太少，所以从负样本中随机采样5000000个
data_1 = data[data['click']==1]
data_2 =data_1.append(data[data['click']==0].sample(n=2000000,random_state=10, axis=0))

data_2.to_csv('train_FE_6.csv',index=False)
